package com.eric.keywords;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.Collection;
import java.util.HashSet;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.eric.downloader.BDSDownloader;
import com.eric.downloader.MyProxy;
import com.eric.seo.serp.BaiduSERPAnalyzer;

public class Digger implements Runnable{
	
	public static LinkedBlockingQueue<String> queue = new LinkedBlockingQueue<>();
	
	public static File file;

	MyProxy p = new MyProxy();
	
	public static void main(String[] args) throws IOException, InterruptedException {
		
		Digger dig = new Digger(args[0],args[1]);
		
		ExecutorService exe = Executors.newCachedThreadPool();
		
		for (int i=0;i<5;i++){
			exe.execute(dig);
		}
		exe.shutdown();
	}
		
	public Digger (String initKw,String fileLoc) {
		queue.add(initKw);
		file = new File(fileLoc);
	}

	@SuppressWarnings("unchecked")
	@Override
	public void run() {
		BaiduSERPAnalyzer bd = new BaiduSERPAnalyzer();
		BDSDownloader bds = new BDSDownloader();
//		ThreeSixtyRecDownloader three = new ThreeSixtyRecDownloader();

		BufferedWriter bw;
		BufferedWriter bw_q;
		try {
			bw = new BufferedWriter(new FileWriter(file));
			bw_q = new BufferedWriter(new FileWriter(new File(file.getAbsolutePath() + "queue")));

			while (!queue.isEmpty()) {
				if (queue.size() > 200000) {
					int size = queue.size();
					for (int i = 5; i < size; i++) {
						bw_q.write(queue.poll());
						bw_q.write("\n");
					}
					bw_q.flush();
				}

				String kw = queue.poll();
				if (kw.equals(" ") || kw.equals(""))
					continue;

				// ȡ��analyze���json���rec��������ʱJSONArray
				JSONArray ja = bd.getRecKw(kw).getJSONArray("rec");
				ja.addAll(bds.analyze(kw).getJSONArray("rec"));
//				ja.addAll(three.analyze(kw).getJSONArray("rec"));

				// ͨ��hashsetȥ��
				HashSet<String> set = new HashSet<>();
				for (int i = 0; i < ja.size(); i++) {
					set.add(ja.getString(i));
				}
				ja.clear();

				// �����ʱJSONArray
				for (String s : set) {
					ja.add(s);
				}

				// �����һ��json��д���ļ�
				JSONObject json = new JSONObject();
				json.put("kw", kw);
				json.put("rec", ja.clone());
				bw.write(json.toJSONString());
				bw.write("\n");
				bw.flush();

				queue.addAll((Collection<? extends String>) set.clone());

				Thread.sleep(5000);
			}
			bw.close();
			bw_q.close();
		} catch (IOException e) {
			e.printStackTrace();
		} catch (InterruptedException e) {
			e.printStackTrace();
		}
	}

}
